#!/bin/bash

function save_env() {
	env > ${TESTLOG_DIR}/${TESTNAME}.debug.env 2>&1
}

function print_var() {
	local message=$1
	local content=$2
	if [ -n "$content" ]; then
		echo "$message $content"
	fi
}

function print_test_info() {
	echo "start $1 testing..."
	print_var "device name: " "$DEVICE_NAME"
	print_var "driver name: " "$DRIVER"
	print_var "module name: " "$MODULE"
	print_var "device path: " "$DEVPATH"
	echo

	if [ -n "$PCI_BUS_DEVICE_FUNCTION" ]; then
		echo "device pci info:"
		lspci -s "$PCI_BUS_DEVICE_FUNCTION"  -vvvvv
		echo
	fi
}

function test_pass() {
	save_env
	echo "$TESTNAME test PASS!"
	exit 0
}

function test_fail() {
	save_env
	echo "$TESTNAME test FAIL!"
	exit 1
}

function test_skip() {
    local exit_msg="$1"
    save_env
    echo "$TESTNAME test SKIP!"
    echo "$exit_msg"
    exit 2
}

function run_cmd() {
    local the_cmd="$1"
    local log=${2:-""}

    if [ -n "$log" ];then
        echo "$the_cmd > ${log}"
        eval "$the_cmd" > ${log}
    else
        echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
        echo "$the_cmd"
        eval "$the_cmd"
        [ $? -ne 0 ] && echo "failed to run $the_cmd" && test_fail
        echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
        echo;echo;echo
    fi
}

function show_module_info() {
	local module=$1
	echo
	echo "module name $module, detail info:"
	if [ -n "$module" ]; then
		modinfo "$module"
	else
		echo "warning: module name is empty!"
	fi
}

function clean_up_disk_partition() {
	local dev_path=$1

	echo "clean up all partitions for $dev_path"
	local mount_folder
	mount_folder="$(mount | egrep "^$dev_path" | awk '{print $3}')"
	for folder in $mount_folder; do
		echo "umount folder $folder"
		umount "$folder"
		if [ $? -ne 0 ]; then
			local mount_pid
			mount_pid="$(lsof | grep -w "$folder" | awk '{print $2}')"
			echo "kill pid $mount_pid"
			kill -9 "$mount_pid"
			sleep 3
			umount "$folder" || { echo "fail to umount $folder"; test_fail; }
		fi
	done

	echo "wipe out device $dev_path"
	wipefs -f -a "$dev_path" || { echo "fail to wipe out $dev_path"; test_fail; }
}

function check_partition_ready() {
	partition=$1
	echo "check partition $partition status"
	sleep 1
	for ((i=0; i < 10; i++)); do
		if ls "$partition" > /dev/null 2>&1; then
			break
		fi
		sleep 1
	done
	echo "partition $partition is ready, partition info:"
	lsblk "$partition"
}

function show_network_info() {
	echo "get all the link information:"
	ifconfig -a

	echo "device path: $DEVPATH"
	local bdf
	bdf="$(echo "$DEVPATH" | awk -F'/' '{print $NF}')"

	echo "device info:"
	lspci -vvvv -s "$bdf"
}

function show_drive_info() {
	local dev_name=${1:-"${DEVNAME}"}
	lsblk "$DEVNAME"
	echo
	fdisk -l "$DEVNAME"
	echo
	mount
	echo
	df -P
}

function check_mount_point() {
	local dev_name=${1:-"${DEVNAME}"}
	echo "check mount point $dev_name"
	if mount | grep -q "$dev_name"; then
		echo "has mount point for $dev_name:"
		mount | grep "$dev_name"
		return 1
	fi
	echo "dose not have mount point for $dev_name"
	return 0
}

function create_ext4_from_raw_disk() {
	local test_dir=$1
	local dev_name=${2:-"${DEVNAME}"}
	local partition_num=${3:-"1"}
	echo "setup partition $dev_name"
	echo -e 'n\np\n\n\n+8G\nwq\np\n' | fdisk "$dev_name"
	check_partition_ready "${dev_name}${partition_num}"
	echo "create ext4 file system on $dev_name"
	mkfs.ext4 -F "${dev_name}${partition_num}"
	echo "make test dir $test_dir"
	mkdir $test_dir
	echo "mount $test_dir"
	mount -t ext4 "${dev_name}${partition_num}" $test_dir
	[ $? -eq 0 ] || { echo "failed to mount $test_dir"; test_fail; }
	check_mount_point "$test_dir"
        echo "check mount point"
	mount | grep "$test_dir"
	echo "ls command check mount dir:"
	ls -l $test_dir
	[ $? -eq 0 ] || { echo "failed to access $test_dir"; test_fail; }
	echo
}

function generate_fio_config_file() {
	local fio_config_file=$1
	local test_dir=$2

cat > "$fio_config_file" <<FIOFILE
[global]
ioengine=sync
iodepth=4
numjobs=1
group_reporting
size=128M
direct=1
runtime=120
timeout=900

[verify-write]
bs=4k
name=iops_seqwrite
filename=${test_dir}/testfio_verifyrw
offset=0x0000
verify=md5
do_verify=1
verify_pattern=0xff
verify_dump=1
verify_fatal=1
verify_backlog=1
verify_interval=4k
rw=write
stonewall

[verify-read]
bs=4k
name=iops_seqread
filename=${test_dir}/testfio_verifyrw
offset=0x0000
verify=md5
do_verify=1
verify_pattern=0xff
verify_dump=1
verify_fatal=1
verify_backlog=1
verify_interval=16k
rw=read
stonewall

[seq-write]
bs=4k
name=iops_seqwrite
filename=${test_dir}/testfio_seqwrite
rw=write
stonewall

[seq-read]
bs=4k
name=iops_seqread
filename=${test_dir}/testfio_seqread
rw=read
stonewall

[rand-read]
bs=16k
name=iops_randread
filename=${test_dir}/testfio_randread
rw=randread
stonewall

[rand-write]
bs=64k
name=iops_randwrite
filename=${test_dir}/testfio_randwrite
rw=randwrite
stonewall

[randrw]
bs=512k
name=iops_randrw
filename=${test_dir}/testfio_randrw
rwmixread=70
rw=randrw
stonewall
FIOFILE
}

function run_fio_test() {
	local test_type=${1:-"storage"}
	local fio_config_file=$TESTLOG_DIR/fio_config_file.$$
	local test_dir=/ancert.${test_type}.fio.$$

	if $IS_RAW_DISK; then
		check_mount_point "$DEVNAME"
		if [ $? -ne 0 ]; then
			echo "failed, $DEVNAME has mount point"
			test_fail
		fi
		echo "$DEVNAME is a free disk"
		if [ "$test_type" == "nvme" ]; then
			create_ext4_from_raw_disk "$test_dir" "$DEVNAME" "p1"
		else
			create_ext4_from_raw_disk "$test_dir" "$DEVNAME"
		fi
	else
		check_mount_point "$ANCERT_TEST_MOUNT_POINT"
		if [ $? -ne 1 ]; then
			echo "failed, dose not have mount point $ANCERT_TEST_MOUNT_POINT"
			test_fail
		fi
		if [ "$ANCERT_TEST_MOUNT_POINT" != "/" ]; then
			rm -rf $ANCERT_TEST_MOUNT_POINT/ancert.${test_type}.fio.*
			test_dir=$ANCERT_TEST_MOUNT_POINT/ancert.${test_type}.fio.$$
		fi
		mkdir -p $test_dir
	fi

	echo
	echo "test dir is $test_dir"
	echo
	generate_fio_config_file "$fio_config_file" "$test_dir"
	echo
	echo "change work dir to $TESTLOG_DIR"
	cd $TESTLOG_DIR
	run_cmd "fio $fio_config_file"
	cd -
	echo "current work dir `pwd`"

	if $IS_RAW_DISK; then
		umount $test_dir
		clean_up_disk_partition "$DEVNAME"
	fi	

	if [ -d $test_dir ]; then
		rm -rf $test_dir
	fi
}

function check_debugfs() {
    if ! cat /proc/mounts | grep -wq debugfs;then
        mount -t debugfs none /sys/kernel/debug
    fi
    cat /proc/mounts | grep -w debugfs| grep -q /sys/kernel/debug
    if [ $? -ne 0 ];then
        echo "Kernel without debugfs support ?"
        return 1
    else
        return 0
    fi
}

function check_cmd() {
    echo "Check command: $1"
    if ! command -v "$1">/dev/null 2>&1;then
        echo "Error: $1 command not found or may not be available!"
        exit 1
    fi
}

function check_dmesg(){
    local msg_start=$1
    local msg_end=$2
    local exec_operate=$3
    local regx_msg=$4
    local cmd="dmesg -l err|awk '/${msg_start}/,/${msg_end}/{print \$0}'"

    echo "Check whether there are dmesg err logs."
    test "$msg_start" = "$msg_end" && { echo "No lateset dmesg log has been generated.";return 0; }
    echo "$cmd |grep -iE \"$regx_msg\""
    if eval "$cmd |grep -iE \"$regx_msg\"";then
        echo "Error: failed to exec $exec_operate, in dmesg err log!"
        return 1
    fi
    return 0
}

function check_nvme_driver_is_ready(){
    local insmod_path=""
    local nvme_driver="nvme"
    echo -e "\nCheck whether the nvme driver is ready."

    if modinfo $nvme_driver >/dev/null 2>&1;then
        insmod_path="$(modinfo $nvme_driver|grep "filename"|awk '{print $NF}')"
    fi

    if ! lsmod |awk '{print $1}'|grep -wq $nvme_driver;then
        if [ -n "$insmod_path" ];then
            insmod "$insmod_path" && { sleep 5;return 0; } || { echo "Failed to exec cmd[insmod $insmod_path].";return 1; }
        else
            echo "Failed to exec cmd[modinfo $nvme_driver]." && return 1
        fi
    else
        lsmod |awk '{print $1}'|grep -w $nvme_driver && return 0 || { echo "Failed to exec it[lsmod|grep nvme].";return 1; }
    fi
}

function nvme_dev_check() {
    echo "Nvme devices check."
    check_cmd "nvme"

    if nvme list | grep -q nvme;then
        NVME_DEVS=`nvme list | grep nvme | awk -F " " '{print $1}'`
        [ $? -ne 0 ] && { echo "nvme cli run failed.";test_fail; }
        # At present, only trim check is performed for nvme devices, not for scsi.
        [ -z "${NVME_DEVS}" ] && { echo "no nvme dev, exit.";test_skip; }
        export NVME_DEVS
    else
        echo "no nvme dev, exit."
        test_skip
    fi
}

function get_nvme_free_disk(){
    check_cmd "nvme"
    local nvme_dev_list="$(nvme list | grep nvme |awk '{print $1}')"
    local nvme_free_disk=""

    if [ -z "$nvme_dev_list" ];then
        if check_nvme_driver_is_ready;then
            nvme_dev_list="$(nvme list | grep nvme |awk '{print $1}')"
            [ -z "$nvme_dev_list" ] && { echo "Error: Without any nvme equipment";return 1; }
        else
            return 1
        fi
    fi

    for nd in ${nvme_dev_list};do
        if echo $DEVNAME|grep -wq "$nd";then
            lsblk "$nd" -no MOUNTPOINT|grep -Ewq '/|/boot|/(.*)' || nvme_free_disk="${nvme_free_disk},${nd}"
        fi
    done
    NVME_FREE_DISK=($(echo $nvme_free_disk|awk '{gsub(/,/," "); print}'))
    if [ ${#NVME_FREE_DISK[*]} -ge 1 ];then
        export NVME_FREE_DISK
        echo "Get nvme free disk: ${NVME_FREE_DISK[*]}"
        return 0
    else
        return 1
    fi
}

function fio_setup() {
    local args=$@
    local fio_filename="${DEVNAME}"
    local fio_rw="write"
    local fio_bs="1M"
    local fio_runtime=120
    local fio_numjobs=8
    local fio_iodepth=64
    local fio_name="fio_write_test"
    local fio_rwmixread=0
    echo "Fio setup for $fio_filename."

    for arg in ${args};do
        case ${arg} in
            filename=*) fio_filename="$(echo $arg|awk -F= '{print $NF}')";;
            rw=*) fio_rw="$(echo $arg|awk -F= '{print $NF}')";;
            bs=*) fio_bs="$(echo $arg|awk -F= '{print $NF}')";;
            runtime=*) fio_runtime="$(echo $arg|awk -F= '{print $NF}')";;
            numjobs=*) fio_numjobs="$(echo $arg|awk -F= '{print $NF}')";;
            iodepth=*) fio_iodepth="$(echo $arg|awk -F= '{print $NF}')";;
            name=*) fio_name="$(echo $arg|awk -F= '{print $NF}')";;
            rwmixread=*) fio_rwmixread="$(echo $arg|awk -F= '{print $NF}')";;
        esac
    done
    export FIO_CMD="fio --filename=${fio_filename} --ioengine=sync --bs=${fio_bs} --rw=${fio_rw} --rwmixread=${fio_rwmixread} --runtime=${fio_runtime} --numjobs=${fio_numjobs} --iodepth=${fio_iodepth} --direct=1 --group_reporting --name=${fio_name}"
}

function is_supported_apei_error_inject() {
    local einj_table="/sys/firmware/acpi/tables/EINJ"
    local kernel_confs="/lib/modules/`uname -r`/config /usr/src/kernels/`uname -r`/.config"
    local config_options="CONFIG_ACPI_APEI_EINJ=m CONFIG_ACPI_APEI=y CONFIG_DEBUG_FS=y"
    local has_kernel_conf="false"
    local check_failed_msg=""
    if test -f $einj_table;then
        echo "`ls -l $einj_table`"
        for kf in $kernel_confs;do
            if test -f $kf;then
                has_kernel_conf="true"
                echo "Check following options in kernel configuration."
                for op in $config_options;do
                    if grep -wqE "$op" "$kf";then
                        echo "`grep -wE "$op" "$kf"`--->${kf}"
                    else
                        check_failed_msg="${check_failed_msg}failed to check ${op} in kernel configuration.\n"
                    fi
                done
                echo "It may supported APEI Error INJection!"
                break
            fi
        done
    else
        echo "Error, $einj_table: No such file or directory!"
        echo "It is not supported APEI Error INJection!"
        return 1
    fi

    if ! "$has_kernel_conf";then
        echo "Error: not found ${kernel_confs}" && return 1
    fi
    [ -n "$check_failed_msg" ] && { echo -e "$check_failed_msg";return 1; } || return 0
}

function cpu_ras_setup() {
    local dmesg_end=""
    local einj_dir="/sys/kernel/debug/apei/einj"
    local dmesg_start=$(dmesg|tail -n1|awk 'gsub(/\[|\]/,"") {print $1}')
    local feature_support_in_arch="aarch64"

    echo "Cpu ras setup."
    if ! echo "$feature_support_in_arch"|grep -wq "$(arch)";then
        echo "Error: CPU RAS feature is not supported In $(arch) architecture." && exit 1
    fi

    check_debugfs
    [ $? -eq 1 ] && return 1
    is_supported_apei_error_inject || exit 1
    echo "Check modinfo einj."
    if modinfo einj >/dev/null 2>&1;then
        modinfo einj
    fi

    if lsmod | grep -w einj;then
        if ! test -d "$einj_dir";then
            echo "Error, $einj_dir: No such directory, please reoperate modprobe einj." && return 1
        else
            echo "------------------------------"
            echo -e "ls $einj_dir\n`ls $einj_dir`"
            echo "------------------------------"
            return 0
        fi
    else
        echo "modprobe einj."
        modprobe einj param_extension=1
        if [ $? -eq 0 ];then
            sleep 5
            dmesg_end=$(dmesg|tail -n1|awk 'gsub(/\[|\]/,"") {print $1}')
            check_dmesg $dmesg_start $dmesg_end "modprobe einj param_extension=1" "EINJ" || exit 1
            if lsmod | grep -q einj;then
                echo "Successfully execute modprobe einj param_extension=1" && return 0
            else
                echo "Error, failed to execute lsmod |grep einj" && return 1
            fi
        else
            dmesg_end=$(dmesg|tail -n1|awk 'gsub(/\[|\]/,"") {print $1}')
            check_dmesg $dmesg_start $dmesg_end "modprobe einj param_extension=1" "EINJ" || exit 1
            echo "Error, failed to modprobe einj!" && return 1
        fi
    fi
}

function clean_einj() {
    echo "Clean einj."
    if lsmod | grep -wq einj;then
        rmmod einj
        [ $? -ne 0 ] && { echo "Failed to rmmod einj.";test_fail; }
    fi
    test_fail
}

function is_ping_pass() {
    local ip=$1
    echo "Ping $ip -c 3"
    if timeout 10 ping "$ip" -c 5 >/dev/null 2>&1;then
        return 0
    else
        echo "Error: connect: Network is unreachable for $ip"
        return 1
    fi
}

function is_secret_free_login() {
    local ip=$1
    check_cmd "expect"

    echo "warning: mkdir/var/log/sssd before ssh connection, Default log path required by the sssd service."
    [ ! -d /var/log/sssd ] && mkdir -p /var/log/sssd

    echo "warning: /root/.ssh/known_hosts are backed up before the ssh connection,move it."
    if test -f /root/.ssh/known_hosts;then
        if test ! -f /root/.ssh/known_hosts.bak;then
            mv /root/.ssh/known_hosts /root/.ssh/known_hosts.bak
        else
            rm -rf /root/.ssh/known_hosts.bak
            mv /root/.ssh/known_hosts /root/.ssh/known_hosts.bak
        fi
    fi

    sh ./../../../utils/sshconf.sh "setup" || test_fail
    echo "Verify the password-free login."
    expect << EOF
    spawn ssh root@${ip}
    expect {
        "yes/no" {send "yes\n"; exp_continue}
        "*password:" {send "password\n"; exp_continue}
        "#" {send "exit 0\n"}
    }
    set timeout -1
    send "exit -1\r"
    expect eof
EOF
    if [ $? -eq 0 ];then
        return 0
    else
        echo "Error: failed to connect $ip"
        echo "Please configure the password-free login first!"
        return 1
    fi
}

function remote_check_firewall_service() {
    local ip=$1

    echo -e "\ncheck firewall service on remote $ip."
    $ESSH -tq root@$ip << "EOF"
    #!/usr/bin/env bash

    function check_firewall_service() {
        local failed_msg=""
        local firewall_status="$(systemctl is-active firewalld)"
        local iptables_status="$(systemctl is-active iptables)"

        echo "check firewall service."
        echo -e "\n[DEFAULT FIREWALL STATUS]: $firewall_status\n[DEFAULT IPTABLES STATUS]: $iptables_status\n"
        if echo "$firewall_status"|grep -wq "active";then
            systemctl stop firewalld || failed_msg="${failed_msg}Failed to stop firewalld service.\n"
            systemctl status firewalld
        fi

        if echo "$iptables_status"|grep -wq "active";then
            systemctl stop iptables || failed_msg="${failed_msg}Failed to stop iptables service.\n"
            systemctl status iptables
        fi
        [ -n "$failed_msg" ] && { echo -e "$failed_msg";exit 1; } || { echo "The firewall service is successfully checked.";exit 0; }
    }
    check_firewall_service
EOF
    [ $? -eq 0 ] && return 0 || return 1
}

function ssh_remote_exec(){
    local cmd=$(echo $1|sed -e 's#"#\\"#g; s#\$#\\$#g')
    local ssh_remote_cmd="$ESSH root@$LTS_IPADDRESS \"${cmd}\""
    echo -e "\n$ssh_remote_cmd"
    if eval $ssh_remote_cmd;then
        return 0
    else
        echo "Error: failed to exec $cmd from remote ${LTS_IPADDRESS}!"
        return 1
    fi
}

function remote_check_iperf3_service() {
    local operate=$1
    local port=${2:-5201}
    local info=""
    echo -e "\nCheck iperf3 service"

    if ! ssh_remote_exec "command -v iperf3 >/dev/null 2>&1";then
        echo "Error: command not find iperf3 or may not be available on the $LTS_IPADDRESS." && exit 1
    fi

    case ${operate} in
    start)
        echo "Iperf3 service start"
        ssh_remote_exec "iperf3 -sD -p $port &"
        sleep 3 ;;
    stop)
        echo "Iperf3 service stop"
        cmd="ps -ef|grep iperf3|grep -vE 'bash|grep'| grep -w $port|awk '{print \$2}'"
        echo "$ESSH -tq root@$LTS_IPADDRESS \"pids=\$(eval $cmd) && [ -n \"\$pids\" ] && kill -9 \$pids||:\""
        pids="$(eval $ESSH -tq root@$LTS_IPADDRESS "$cmd")"
        [ X"$pids" != X"" ] && eval $ESSH -tq root@$LTS_IPADDRESS "kill -9 $pids"
         ;;
    service)
        echo "Check iperf3 service"
        echo "$ESSH root@$LTS_IPADDRESS \"ps -ef |grep -vE 'bash|grep'|grep -w 'iperf3 -sD'|grep -w $port\""
        $ESSH root@$LTS_IPADDRESS "ps -ef |grep -vE 'bash|grep'|grep -w 'iperf3 -sD'|grep -w $port"
        [ $? -eq 0 ] && return 0 || return 1 ;;
    *)
        echo "Error: unrecognized option,please select it.(start|stop|service)"
        return 1 ;;
    esac
}

function get_online_use_network_interface() {
    local online_net_face=""
    local num=0
    local cur_sum_traffic=0
    local cur_traffic=0
    local cur_net_face=""
    echo -e "\nGet online use network interface."
    while read line;do
        ((num++))
        [ "$num" -le 2 ] && continue
        cur_traffic=$[$(echo $line|awk '{print $2"+"$10}')]
        cur_net_face=$(echo $line|awk -F: '{print $1}')
        if [ $(echo "$cur_traffic > $cur_sum_traffic"|bc -l) -eq 1 ];then
            cur_sum_traffic=$cur_traffic
            online_net_face=$cur_net_face
        fi
    done </proc/net/dev
    if [ -n "$online_net_face" ];then
        export ONLINE_NET_FACE=$online_net_face
        echo "Online use network interface: $online_net_face"
        return 0
    else
        return 1
    fi
}

function get_available_interface(){
    local is_filter_ssh_interface=${1:-true}
    local link_status='Link detected: yes'
    local test_net_arr=()
    local netinfo=$(ls /sys/class/net/|grep -vE "bond*|lo")

    echo -e "\nGet available interface(linked and Off-line use)."
    $is_filter_ssh_interface && { get_online_use_network_interface || { echo "Error: No network card is available online";exit 1; }; }
    $is_filter_ssh_interface && netinfo=$(echo $netinfo|grep -v "$ONLINE_NET_FACE")
    for interface in ${netinfo};do
        echo $INTERFACES |grep -wq $interface || continue
        if ethtool $interface|grep -wq "$link_status";then
            echo "$interface is linked"
            test_net_arr[${#test_net_arr[@]}]=$interface
        else
            echo "$interface is not linked"
        fi
    done

    if [ ${#test_net_arr[*]} -ge 1 ];then
        echo "test net interface : ${test_net_arr[@]}"
        export AVAILABLE_INTERFACE=${test_net_arr[@]}
        return 0
    else
        return 1
    fi
}

function write_messages() {
    local level_info=$1
    local messages=$2
    local cur_date=$(date "+%F %H:%M:%S")
    local log_file="log_$(date "+%Y%m%d")"

    [ ! -d "${TESTLOG_DIR}/log" ] && mkdir -p "${TESTLOG_DIR}/log"
    case ${level_info} in
    info) echo -e "\033[1;32m${messages}\033[0m"
	      echo "[${cur_date}][info]${messages}" >> ${TESTLOG_DIR}/log/"${log_file}"
		   ;;
    warn) echo -e "\033[1;33m${messages}\033[0m"
	      echo "[${cur_date}][warn]${messages}" >> ${TESTLOG_DIR}/log/"${log_file}"
		   ;;
    err) echo -e "\033[1;31m${messages}\033[0m"
	     echo "[${cur_date}][warn]${messages}" >> ${TESTLOG_DIR}/log/"${log_file}"
		  ;;
    noecho) echo "[${cur_date}][debug]${messages}" >> ${TESTLOG_DIR}/log/"${log_file}"
	        ;;
    esac
}
